{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入相关的包和加载示例数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>class</th>\n",
       "      <th>who</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>deck</th>\n",
       "      <th>embark_town</th>\n",
       "      <th>alive</th>\n",
       "      <th>alone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Cherbourg</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked  class  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S  Third   \n",
       "1         1       1  female  38.0      1      0  71.2833        C  First   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Third   \n",
       "3         1       1  female  35.0      1      0  53.1000        S  First   \n",
       "4         0       3    male  35.0      0      0   8.0500        S  Third   \n",
       "\n",
       "     who  adult_male deck  embark_town alive  alone  \n",
       "0    man        True  NaN  Southampton    no  False  \n",
       "1  woman       False    C    Cherbourg   yes  False  \n",
       "2  woman       False  NaN  Southampton   yes   True  \n",
       "3  woman       False    C  Southampton   yes  False  \n",
       "4    man        True  NaN  Southampton    no   True  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "#seaborn里有数据集，可以直接加载使用\n",
    "titanic_df = sns.load_dataset('titanic')\n",
    "#查看前5条的数据\n",
    "titanic_df.head() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查看数据量和概览"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(891, 15)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 15 columns):\n",
      "survived       891 non-null int64\n",
      "pclass         891 non-null int64\n",
      "sex            891 non-null object\n",
      "age            714 non-null float64\n",
      "sibsp          891 non-null int64\n",
      "parch          891 non-null int64\n",
      "fare           891 non-null float64\n",
      "embarked       889 non-null object\n",
      "class          891 non-null category\n",
      "who            891 non-null object\n",
      "adult_male     891 non-null bool\n",
      "deck           203 non-null category\n",
      "embark_town    889 non-null object\n",
      "alive          891 non-null object\n",
      "alone          891 non-null bool\n",
      "dtypes: bool(2), category(2), float64(2), int64(4), object(5)\n",
      "memory usage: 80.6+ KB\n"
     ]
    }
   ],
   "source": [
    "print(titanic_df.shape)\n",
    "#输出 （891，15）\n",
    "#查看DataFrame的基本信息，包含索引、字段名称、非空值统计、字段类型。\n",
    "titanic_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查看数值型变量的描述统计信息，数量、均值、标准差、最大最小值、分位数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>714.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>14.526497</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         survived      pclass         age       sibsp       parch        fare\n",
       "count  891.000000  891.000000  714.000000  891.000000  891.000000  891.000000\n",
       "mean     0.383838    2.308642   29.699118    0.523008    0.381594   32.204208\n",
       "std      0.486592    0.836071   14.526497    1.102743    0.806057   49.693429\n",
       "min      0.000000    1.000000    0.420000    0.000000    0.000000    0.000000\n",
       "25%      0.000000    2.000000   20.125000    0.000000    0.000000    7.910400\n",
       "50%      0.000000    3.000000   28.000000    0.000000    0.000000   14.454200\n",
       "75%      1.000000    3.000000   38.000000    1.000000    0.000000   31.000000\n",
       "max      1.000000    3.000000   80.000000    8.000000    6.000000  512.329200"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titanic_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 条形图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.countplot(titanic_df[\"embarked\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/5.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 饼图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#先对类别变量'embarked'进行聚合计算各类别频数\n",
    "embarked_cnt=titanic_df.groupby('embarked')['embarked'].count()\n",
    "#使用pandas 接口画饼图\n",
    "embarked_cnt.plot.pie(autopct='%1.2f%%', figsize=(6, 6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/6.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 直方图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#输入空值会报错需要剔除空值\n",
    "sns.distplot(titanic_df[titanic_df.age.isnull().values == False]['age'],kde=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/7.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 箱型图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(titanic_df['age'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/8.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 小提琴图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y=titanic_df['age'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/9.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 散点图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#查看年龄和舱位等级散点图\n",
    "sns.swarmplot(y='age', x='pclass', data=titanic_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/10.jpg)\n",
    "通过散点图可以直观地发现，买得起头等舱(pclass=1)的年龄相对大一些，普通舱(pclass=2)则以青年为主，符合我们的认知。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 相关性图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr\n",
    "#查看年龄和舱位等级相关性\n",
    "sns.jointplot(x=\"pclass\", y=\"age\", data=titanic_df, \n",
    "kind=\"reg\",stat_func=pearsonr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 堆积柱形图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = titanic_df.groupby(['embarked', 'who']).who.count()\n",
    "var.unstack().plot(kind='bar', stacked=True, color=['red', 'blue', 'green'], figsize=(8, 6))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/12.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 卡方检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "卡方值：26.48914983923762\n",
      "p_value：1.769922284120912e-06\n",
      "自由度：2\n",
      "期望频率：[[103.7480315  64.2519685]\n",
      " [ 47.5511811  29.4488189]\n",
      " [397.7007874 246.2992126]]\n"
     ]
    }
   ],
   "source": [
    "#导入scipy.stats.chi2_contingency包\n",
    "from scipy.stats import chi2_contingency\n",
    "#计算embarked和alive交叉列联表 \n",
    "embarked_alive = pd.crosstab(titanic_df.embarked, titanic_df.alive)\n",
    "chi2, p, dof, ex = chi2_contingency(embarked_alive)\n",
    "print('卡方值：{}'.format(chi2))\n",
    "print('p_value：{}'.format(p))\n",
    "print('自由度：{}'.format(dof))\n",
    "print('期望频率：{}'.format(ex))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 带误差条的线图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.errorbar(x=titanic_df.groupby([ 'class']).age.mean().index,\n",
    "             y=titanic_df.groupby(['class']).age.mean(),\n",
    "             yerr=titanic_df.groupby(['class']).age.std(),\n",
    "             fmt=\"o\")\n",
    "\n",
    "plt.xlabel('class')\n",
    "plt.ylabel('age')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/13.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 组合图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = sns.boxplot(x=\"alive\", y=\"age\", data=titanic_df)\n",
    "ax = sns.swarmplot(x=\"alive\", y=\"age\", data=titanic_df, color=\".8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/14.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Z检验和t检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t值：-2.499206354920835\n",
      "p_value：0.012671296797013709\n"
     ]
    }
   ],
   "source": [
    "#导入scipy.stats.ttest_ind\n",
    "from scipy.stats import ttest_ind\n",
    "#剔除age空值 \n",
    "titanic_age=titanic_df[titanic_df.age.isnull().values == False]\n",
    "\n",
    "t_statistics, p = ttest_ind(titanic_age[titanic_age['sex'] == 'female'].age,\n",
    "                            titanic_age[titanic_age['sex'] == 'male'].age)\n",
    "\n",
    "print('t值：{}'.format(t_statistics))\n",
    "print('p_value：{}'.format(p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 多变量可视化分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入数据\n",
    "titanic = sns.load_dataset('titanic')\n",
    "#对缺失值进行简单的填充\n",
    "titanic['age'].fillna(titanic['age'].median(), inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 多图网格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.FacetGrid(titanic, hue=\"alive\", size=6).map(plt.scatter, \"age\", \"fare\").add_legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/15.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### PairGrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#为了图的展示效果，行列上只选了两个字段，实际使用可以选择多个字段\n",
    "g = sns.PairGrid(titanic[['age', 'fare', 'who']], hue=\"who\", size=5)\n",
    "g.map_diag(plt.hist)\n",
    "g.map_offdiag(plt.scatter)\n",
    "g.add_legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/16.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Heatmap热力图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#变量间相关系数热力图\n",
    "f = titanic[['age', 'fare', 'sibsp']].corr()\n",
    "sns.heatmap(f, annot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/17.jpg)\n",
    "图中的数值是皮尔森相关系数，浅颜色表示相关性高。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##    查看缺失情况"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Missingno缺失值可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import missingno as msno\n",
    "#为了展示效果，只选取部分字段，调整了图片和字段大小\n",
    "msno.matrix(\n",
    "    df=titanic_df[['sex', 'age', 'fare', 'embarked', 'deck', 'embark_town']],\n",
    "    figsize=(8, 4),\n",
    "    fontsize=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/18.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "msno.heatmap(df=titanic_df, figsize=(8, 4), fontsize=18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/19.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "msno.bar(df=titanic_df[['sex', 'age', 'fare', 'embarked',\n",
    "                        'deck', 'embark_town']], figsize=(8, 6), fontsize=18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/20.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 缺失值处理方式"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 删除"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic_df.shape\n",
    "#输出\n",
    "（891,15）\n",
    "\n",
    "#如果行数据中空值，按行删除，按列删除配置axis=1\n",
    "titanic_df_row  = titanic_df.dropna ( axis=0) \n",
    "titanic_df_row.shape\n",
    "#输出\n",
    "（182,15）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_nan_stat(df, copy=False, axis=0, nan_threshold=0.9):\n",
    "    '''按行、列的缺失值比例删除大于缺失值阈值的行、列'''\n",
    "    assert isinstance(df, pd.DataFrame)\n",
    "    return_df = df.copy() if copy else df\n",
    "    n_rows, n_cols = return_df.shape\n",
    "\n",
    "    if axis == 0:\n",
    "        t = return_df.isnull().sum(axis=0)\n",
    "        t = pd.DataFrame(t, columns=['NumOfNan'])\n",
    "        t['PctOFNan'] = t['NumOfNan'] / n_rows\n",
    "        return_df = return_df.drop(\n",
    "            labels=t[t.PctOFNan > nan_threshold].index.tolist(), axis=1)\n",
    "    elif axis == 1:\n",
    "        t = return_df.isnull().sum(axis=1)\n",
    "        t = pd.DataFrame(t, columns=['NumOfNan'])\n",
    "        t['PctOFNan'] = t['NumOfNan'] / n_cols\n",
    "        print(t)\n",
    "        return_df = return_df.drop(\n",
    "            labels=t[t.PctOFNan > nan_threshold].index.tolist(), axis=0)\n",
    "\n",
    "    return return_df\n",
    "\n",
    "#删除缺失值比例大于0.5的列\n",
    "titanic_df_col = drop_nan_stat(df=titanic_df,\n",
    "                               copy=True,\n",
    "                               axis=0,\n",
    "                               nan_threshold=0.5)\n",
    "msno.bar(df=titanic_df_col, figsize=(8, 4), fontsize=18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/21.jpg)\n",
    "可以发现缺失值比例大于0.5的列“deck”已经被删除。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 填充"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 替换缺失值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#由于后续的案例也会使用titanic_df，保持titanic_df数据不变，复制一份新数据进行填充操作。\n",
    "titanic_df_fill=titanic_df.copy()\n",
    "titanic_df_fill.info()\n",
    "\n",
    "titanic_df_fill['age'].fillna(titanic_df_fill['age'].median(), inplace=True)\n",
    "#判断age填充后是否还有缺失值\n",
    "titanic_df_fill['age'].isnull().any()\n",
    "#输出\n",
    "False\n",
    "\n",
    "#类似的，类别变量'embarked'使用众数填充示例：\n",
    "titanic_df_fill['embarked'].fillna(titanic_df_fill['embarked'].mode()[0],inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 拟合缺失值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入sklearn.ensemble.RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# RandomForestRegressor只能处理数值、数据，获取缺失值年龄和数值类型变量\n",
    "age_df = titanic_df[['age', 'fare', 'parch', 'sibsp', 'pclass']].copy()\n",
    "print(age_df['age'].isnull().any())\n",
    "#输出 True\n",
    "\n",
    "# 按年龄是否缺失，可分为训练数据集和预测数据集\n",
    "train_df = age_df[age_df.age.notnull()].as_matrix()\n",
    "predict_df = age_df[age_df.age.isnull()].as_matrix()\n",
    "# y即目标年龄\n",
    "y = train_df[:, 0]\n",
    "# X即特征属性值\n",
    "X = train_df[:, 1:]\n",
    "\n",
    "# 训练数据集使用RandomForestRegressor训练模型\n",
    "rf_model = RandomForestRegressor(random_state=42, n_estimators=100)\n",
    "rf_model.fit(X, y)\n",
    "# 用训练好的模型预测数据集的年龄进行预测\n",
    "predict_ages = rf_model.predict(predict_df[:, 1:])\n",
    "\n",
    "# 预测结果填补原缺失数据\n",
    "age_df.loc[(age_df.age.isnull()), 'age'] = predict_ages\n",
    "print(age_df['age'].isnull().any())\n",
    "#输出\n",
    "False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 虚拟变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_df['age'] = titanic_df['age'].copy()\n",
    "\n",
    "#判断年龄是否缺失，衍生一个新变量age_nan\n",
    "age_df.loc[(age_df.age.notnull()), 'age_nan'] = \"False\"\n",
    "age_df.loc[(age_df.age.isnull()), 'age_nan'] = \"True\"\n",
    "\n",
    "#统计新变量age_nan缺失和非缺失的数量，可以与之前的缺失值可视化进行缺失值数据核验\n",
    "age_df['age_nan'].value_counts()\n",
    "#输出\n",
    "False    714\n",
    "True     177"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异常值检测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#只选取上述泰坦尼克数据集的3个字段作为案例展示\n",
    "columns=['pclass','age','fare']\n",
    "for var in columns:\n",
    "    titanic_df[var + '_zscore'] = (titanic_df[var] - titanic_df[var].mean()) / titanic_df[var].std()\n",
    "    z_normal = abs(titanic_df[var + '_zscore']) > 3\n",
    "    print(var + '中有' + str(z_normal.sum()) + '个异常值')\n",
    "#输出\n",
    "pclass中有0个异常值\n",
    "age中有2个异常值\n",
    "fare中有20个异常值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for var in columns:\n",
    "    iqr = titanic_df[var].quantile(0.75) - titanic_df[var].quantile(0.25)\n",
    "    q_abnormal_L = titanic_df[var] < titanic_df[var].quantile(0.25) - 1.5 * iqr\n",
    "    q_abnormal_U = titanic_df[var] > titanic_df[var].quantile(0.75) + 1.5 * iqr\n",
    "    print(var + '中有' + str(q_abnormal_L.sum() + q_abnormal_U.sum()) + '个异常值')\n",
    "#输出\n",
    "pclass中有0个异常值\n",
    "age中有11个异常值\n",
    "fare中有116个异常值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入sklearn.cluster.KMeans\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "#修改matplotlib配置参数支持中文显示，\n",
    "plt.rcParams['font.family']='SimHei'\n",
    "\n",
    "#聚类的类别\n",
    "k = 3 \n",
    "#异常点阈值\n",
    "threshold = 3 \n",
    "#读取已经填充过空值的数据\n",
    "data = titanic[['pclass','age','fare']].copy() \n",
    "#数据标准化\n",
    "data_zs = 1.0*(data - data.mean())/data.std() \n",
    "\n",
    "#使用聚类模型聚类\n",
    "model = KMeans(n_clusters = 3, max_iter = 500) \n",
    "model.fit(data_zs) \n",
    " \n",
    "#标准化数据及其类别\n",
    "r = pd.concat([data_zs, pd.Series(model.labels_, index = data.index)], axis = 1)  \n",
    "r.columns = list(data.columns) + ['聚类类别'] \n",
    "\n",
    "#计算相对距离\n",
    "norm = []\n",
    "for i in range(k): \n",
    "    norm_tmp = r[['pclass','age','fare']][r['聚类类别'] == i]-model.cluster_centers_[i]\n",
    "    norm_tmp = norm_tmp.apply(np.linalg.norm, axis = 1) \n",
    "    norm.append(norm_tmp/norm_tmp.median()) \n",
    "norm = pd.concat(norm)\n",
    "\n",
    "#正常点，相对距离小于等于异常点阈值\n",
    "norm[norm <= threshold].plot(style = 'go')\n",
    "#异常点，相对距离大于异常点阈值\n",
    "discrete_points = norm[norm > threshold] \n",
    "discrete_points.plot(style = 'ro')\n",
    "    \n",
    "plt.xlabel('编号')\n",
    "plt.ylabel('相对距离')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](./imgs/22.jpg)\n",
    "获取大于异常点阈值的索引，方便后续处理.\n",
    "\n",
    "discrete_points = norm[norm>threshold]\n",
    "\n",
    "discrete_points.index"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "577px",
    "left": "24px",
    "top": "110px",
    "width": "287.997px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
